package com.cqw.webauto.rpc;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;

/**
 * 网站抓取文本
 */
public class JsoupWebText {


    public static void main(String[] args) {
        getNewsText();
    }

    //央视新闻 地址
    static final String uri = "https://news.cctv.com/society/?spm=C94212.PBZrLs0D62ld.0.0";

    /**
     * 抓取新闻标题
     * @return
     */
    public static String getNewsText() {
        StringBuilder allText = new StringBuilder();
            // 建立连接，获取网页内容
            String html = Connect(uri );
            // 将内容转换成dom格式，方便操作
            Document doc = Jsoup.parse(html);
            // 获取网页内所有标题节点
            Elements titles = doc.select(".right_text h3");
            for (Element titleEle : titles) {
                // 标题对应的正文
                String content = titleEle.select("a").text();
                String contentUri = titleEle.select("a").attr("href");
                allText.append("[央视新闻] " + content + ".详情：" + contentUri );
                System.out.println(allText);
            }
       return allText.toString();
    }


    /**
     * 建立http连接
     */
    public static String Connect(String address) {
        HttpURLConnection conn = null;
        URL url = null;
        InputStream in = null;
        BufferedReader reader = null;
        StringBuffer stringBuffer = null;
        try {
            url = new URL(address);
            conn = (HttpURLConnection) url.openConnection();
            conn.setConnectTimeout(5000);
            conn.setReadTimeout(5000);
            conn.setDoInput(true);
            conn.connect();
            in = conn.getInputStream();
            reader = new BufferedReader(new InputStreamReader(in));
            stringBuffer = new StringBuffer();
            String line = null;
            while ((line = reader.readLine()) != null) {
                stringBuffer.append(line);
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            conn.disconnect();
            try {
                in.close();
                reader.close();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        return stringBuffer.toString();
    }
}
